#coding:utf-8
import random
import urllib2
from urllib2 import URLError
from urllib2 import HTTPError
import os
from utils import clean_html,segment
from sklearn import linear_model
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import joblib
import sys

user_agent_list = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0) Gecko/20121026 Firefox/16.0',
'Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
'Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1'
]

def http_client (url):
    try:
        rand_num = random.randint(0,29)
        send_headers = {
            'User-Agent':user_agent_list[rand_num],
        }
        req = urllib2.Request(url,headers=send_headers)
        response=urllib2.urlopen(req,data=None, timeout=5)
        return response.read()
    except URLError, e:
        print "can not reach a server,writing..."
        print "write url success!"
        return None
    except HTTPError,code:
        print 'writeurl success'
        return None
    except Exception,e:
        print response.code
        print 'write url success'
        return None

def http_client_selenium(url):
    driver = webdriver.PhantomJS(executable_path='C:/phantomjs/bin/phantomjs.exe')
    try:
        driver.get(url)
    except Exception,e:
        print e
        return None

    return driver.page_source


def craw_html():
    du_files = os.listdir('./du')
    nor_files = os.listdir('./urls2')

    for f in du_files:
        f = open('./du/' + f)
        lines = f.readlines()
        count = 0
        for line in lines:
            url = line.strip('\r\n')
            html = http_client(url)
            if html:
                f_html = open('./duhtml/' + str(count) + '.html','w')
                if type(html) == unicode:
                    html = html.encode('utf-8')
                    f_html.write(html)
                    f_html.close()
                    count += 1

    for f in nor_files:
        f = open('./urls2/' + f)
        lines = f.readlines()
        count = 0
        for line in lines:
            url = line.strip('\r\n')
            html = http_client(url)
            if html:
                f_html = open('./nohtml/' + str(count) + '.html','w')
                if type(html) == unicode:
                    html = html.encode('utf-8')
                    f_html.write(html)
                    f_html.close()
                    count += 1

def get_dataset():
    dudir = './duhtml/'
    nodir = './nohtml/'
    files = os.listdir(dudir.decode('utf-8'))
    all_seg = []
    labels = []
    for file in files:
        if file[-4:] == '.htm':
            f = open(dudir + file).read()
            content = clean_html(f)
            seg = segment(content)
            all_seg.append(seg)
            labels.append(1)
    files = os.listdir(nodir.decode('utf-8'))
    for file in files:
        if file[-4:] == 'html':
            f = open(nodir + file).read()
            content = clean_html(f)
            seg = segment(content)
            all_seg.append(seg)
            labels.append(0)
    length = len(all_seg)
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(all_seg)
    X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.2, random_state=42)
    return X_train, X_test, y_train, y_test,vectorizer


def train_model():
    x_train,x_test,y_train,y_test,vector = get_dataset()
    l_model = LogisticRegression()
    l_model.fit(x_train,y_train)
    print(l_model.score(x_test, y_test))
    return l_model,vector

if __name__ == '__main__':
    url = sys.argv[1]
    content = http_client(url)
    x_predict = []
    #model,vector = train_model()
    #joblib.dump(vector,'tfidf.m')
    #joblib.dump(model,'clf.m')
    vector = joblib.load('tfidf.m')
    model = joblib.load('clf.m')
    #new = open('8.html').read()
    content = clean_html(content)
    seg = segment(content)
    x_predict.append(seg)
    #new = open('9.htm').read()
    #content = clean_html(new)
    #seg = segment(content)
    #x_predict.append(seg)
    x_predict = vector.transform(x_predict)
    y = model.predict(x_predict)
    print y
